# coding: utf-8


import time
import threading
import hashlib
import os
from concurrent import futures
import shutil
from tkinter import Canvas, Tk, StringVar, Label, Button, NW, \
                     GROOVE, scrolledtext, Radiobutton, LabelFrame, ttk
import tkinter as tk
import tkinter.messagebox as msgbox
import re


class MainWin:
    def __init__(self, mywin):
        self.signal = False
        self.time_step = 0.5    # for progress bar
        self.time = time.time()
        self.run_state = ''
        self.bak_result = False

        self.task = 'test'
        self.check_path = 'e:/test'
        self.move_path = self.task + '_move_dir'
        self.move_suffix = 'pdf'
        self.move_option = '1-move'     # [1-copy only, 2- copy and del, 3-del only]
        self.move_num = -1              # move group number to avoid to use too much time at one time

        self.findfiler = None
        self.findmd5 = None
        self.findgroup = None
        self.findsame = None

        self.result1_file = None
        self.result2_md5 = None
        self.result3_group = None
        self.result4_same = None
        self.result5_fail = None

        # mywin = Tk()
        mywin.title('Check Same Files')
        mywin.geometry('950x500+290+100')
        mywin.config(bg='#535363')
        mywin.resizable = (True, True)
        self.mywin = mywin
        self.win_width = 950
        self.win_height = 500

        self.init_pbar(mywin)
        self.init_button(mywin)
        self.inti_entry(mywin)
        self.init_text(mywin)

    # init interface
    def init_pbar(self, mywin):
        # 进度条
        self.bar_length = self.win_width - 10
        self.canvas_progress_bar = Canvas(mywin, width=self.bar_length, height=20)
        self.canvas_shape = self.canvas_progress_bar.create_rectangle(0, 0, 0, 25, fill='green')
        self.canvas_text = self.canvas_progress_bar.create_text(292, 4, anchor=NW)
        self.canvas_progress_bar.itemconfig(self.canvas_text, text='00:00:00')

        self.var_progress_bar_percent = StringVar()
        self.var_progress_bar_percent.set('00.00 %')
        label_progress_bar_percent = Label(mywin,
                                           textvariable=self.var_progress_bar_percent,
                                           fg='#F5F5F5', bg='#535353')

        self.canvas_progress_bar.place(x=5, y=self.win_height-30, width=self.bar_length)
        label_progress_bar_percent.place(x=5, y=self.win_height - 60)

    def init_button(self, mywin):
        # 按钮-1
        button1_start = Button(mywin, text='1: find', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=1),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button1_start.place(relx=0.45, rely=0.5, anchor=tk.SE)

        # 按钮-2
        button2_start = Button(mywin, text='2: md5', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=2),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button2_start.place(relx=0.45, rely=0.5, anchor=tk.SW)

        # 按钮-3
        button3_start = Button(mywin, text='3: group', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=3),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button3_start.place(relx=0.45, rely=0.5, anchor=tk.NE)

        # 按钮-4
        button4_start = Button(mywin, text='4: result', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=4),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button4_start.place(relx=0.45, rely=0.5, anchor=tk.NW)

        # 按钮-5
        button5_start = Button(mywin, text='5: move', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=5),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button5_start.place(relx=0.45, rely=0.5, anchor=tk.NW)

        # 按钮-6
        button6_start = Button(mywin, text=' bakup ', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=6),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button6_start.place(relx=0.45, rely=0.5, anchor=tk.NW)

        # 按钮-7
        button7_start = Button(mywin, text=' exit ', fg='#F5F5F5', bg='#7A7A7A',
                                    command=lambda: self.run_button(step=-1),
                                    height=1, width=15, relief=GROOVE, bd=2,
                                    activebackground='#F5F5F5', activeforeground='#535353')
        button7_start.place(relx=0.45, rely=0.5, anchor=tk.NW)

        button1_start.grid(row=2, column=1, padx=10, pady=10)
        button2_start.grid(row=2, column=2, padx=10, pady=10)
        button3_start.grid(row=2, column=3, padx=10, pady=10)
        button4_start.grid(row=2, column=4, padx=10, pady=10)
        button5_start.grid(row=2, column=5, padx=10, pady=10)
        button6_start.grid(row=2, column=6, padx=10, pady=10)
        button7_start.grid(row=2, column=7, padx=10, pady=10)

    def inti_entry(self, mywin):
        # task label-entry
        label_task = tk.Label(mywin, text='task', bg='#535363', fg='#F5F5F5')
        self.entry_task = tk.Entry(mywin, bd=5)
        self.entry_task.insert(tk.INSERT, self.task)
        # path label-entry
        label_path = tk.Label(mywin, text='path', bg='#535363', fg='#F5F5F5')
        self.entry_path = tk.Entry(mywin, bd=5)
        self.entry_path.insert(tk.INSERT, self.check_path)
        # path label-entry
        label_path_move = tk.Label(mywin, text='move-path', bg='#535363', fg='#F5F5F5')
        self.entry_path_move = tk.Entry(mywin, bd=5)
        self.entry_path_move.insert(tk.INSERT, self.move_path)
        # bak-option label-combobox
        label_bak = tk.Label(mywin, text='bakresult', bg='#535363', fg='#F5F5F5')
        BakComsList = ['1-do not bakup', '2-bakup result']
        self.bak_coms = ttk.Combobox(mywin, width=15, values=BakComsList)
        self.bak_coms.current(0)
        # move-option label-combobox
        label_move_option = tk.Label(mywin, text='move option', bg='#535363', fg='#F5F5F5')
        MoveComsList = ['1-copy longest', '2-copy and del', '3-del but longest']
        self.coms_move_option = ttk.Combobox(mywin, width=15, values=MoveComsList)
        self.coms_move_option.current(0)
        # path label-entry
        label_suffix_move = tk.Label(mywin, text='move-suffix', bg='#535363', fg='#F5F5F5')
        self.entry_suffix_move = tk.Entry(mywin, bd=5)
        self.entry_suffix_move.insert(tk.INSERT, self.move_suffix)

        # place
        left_x = 15
        width = 110
        label_task.place(x=left_x+5, y=43)
        self.entry_task.place(x=left_x, y=65, width=width)
        left_x += width + 15
        width = 175
        label_path.place(x=left_x+5, y=43)
        self.entry_path.place(x=left_x, y=65, width=width)
        left_x += width + 15
        width = 110
        label_bak.place(x=left_x+5, y=45)
        self.bak_coms.place(x=left_x, y=68, width=width)
        left_x += width + 15
        width = 110
        label_move_option.place(x=left_x+5, y=45)
        self.coms_move_option.place(x=left_x, y=68, width=width)
        left_x += width + 15
        width = 120
        label_path_move.place(x=left_x+5, y=43)
        self.entry_path_move.place(x=left_x, y=65, width=width)
        left_x += width + 15
        width = 80
        label_suffix_move.place(x=left_x+5, y=43)
        self.entry_suffix_move.place(x=left_x, y=65, width=width)

    def init_text(self, mywin):
        # result text
        self.text_result = tk.scrolledtext.ScrolledText(mywin, width=129, height=25)
        self.text_result.place(x=12, y=105)
        self.disp_msg(Document.help_doc)

    # run functions
    def run_button(self, step=1):
        if len(self.run_state) > 0:
            msgbox.showwarning('Running Busy Warning',
                               '{} is running'.format(self.run_state))
            return

        # set parameters
        self.task = self.entry_task.get()
        self.check_path = self.entry_path.get()
        self.bak_result = True if self.bak_coms.get()[0] == '2' else False
        self.move_option = self.coms_move_option.get()[0]
        self.move_suffix = self.entry_suffix_move.get()
        self.move_path = self.entry_path_move.get()

        self.text_result.delete('1.0', 'end')

        th = None
        if step == 1:
            # get files
            th = threading.Thread(target=self.run_prog1)
        elif step == 2:
            # calc md5, size
            th = threading.Thread(target=self.run_prog2)
        elif step == 3:
            # group files by md5
            th = threading.Thread(target=self.run_prog3)
        elif step == 4:
            # check same file group
            th = threading.Thread(target=self.run_prog4)
        elif step == 5:
            # move
            th = threading.Thread(target=self.run_prog5)
        elif step == 6:
            # backup
            th = threading.Thread(target=self.run_prog6)
        elif step == -1:
            self.mywin.destroy()
            return
        else:
            self.disp_msg('error button value={}!'.format(step))
        self.run_state = 'step-{} run'.format(step)
        th.setDaemon(True)
        th.start()

    # find subpath and files
    def run_prog1(self):
        self.check_path = self.entry_path.get()
        if not os.path.isdir(self.check_path):
            msgbox.showinfo('step-1 find files from path',
                            'Invalid path={}'.format(self.check_path))
            self.run_state = ''
            return
        self.findfiler = FileFinder(self.check_path)
        finder = self.findfiler

        self.time = time.time()
        self.text_result.insert(tk.INSERT, '='*60+'\n')
        with futures.ThreadPoolExecutor() as executor:
            to_do = []
            future_dict = dict()
            future1 = executor.submit(finder.run_subdir_files)
            to_do.append(future1)
            future_dict.update({future1: 1})
            self.text_result.insert(tk.INSERT, 'step-1 process-{} start ... \n'.format(1))

            self.signal = False
            future2 = executor.submit(self.run_prog1_update_bar, 'step-1')
            to_do.append(future2)
            future_dict.update({future2: 2})
            self.text_result.insert(tk.INSERT, 'bar process-{} start ... \n'.format(2))

            for future in futures.as_completed(to_do):
                if future_dict[future] == 1:
                    result = future.result()
                    self.findfiler.find_file_list = result
                    self.signal = True
                    self.text_result.insert(tk.INSERT, 'find files process-{} end\n'.format(future_dict[future]))
                    self.text_result.insert(tk.INSERT, '-'*60+'\n')
                    [self.text_result.insert(tk.INSERT, str(f)+'\n') for i, f in enumerate(result) if i < 5]
                    self.text_result.insert(tk.INSERT, '-'*60+'\n')
                    end_str = 'check files num={}\nelapsed={:.3f}sec\nrunning end\n'.\
                        format(len(self.findfiler.find_file_list), time.time() - self.time)
                    self.text_result.insert(tk.INSERT, end_str+'='*60+'\n')
            self.result1_file = self.findfiler.find_file_list
            self.write_result(task=self.task,
                              step=1,
                              result_seq=self.result1_file)
        if self.bak_result:
            self.bak_result_fun(1)
        self.text_result.insert(tk.INSERT, 'step-1 end\n{}\n'.format('='*60))
        self.run_state = ''

    # run in thread for dispaying progress or running message
    def run_prog1_update_bar(self, name):
        percent = 0
        while True:
            time.sleep(self.time_step)
            used_time = int(time.time() - self.time)
            percent += 1
            self.update_progress_bar(percent, used_time, name=name)
            if self.signal:
                percent = 0
                self.update_progress_bar(percent, used_time, name='finished, elapsed=')
                break

    def run_prog2(self):
        self.result1_file = self.read_result(self.task, step=1)
        if self.result1_file is None:
            self.run_state = ''
            return
        self.text_result.insert(tk.INSERT, '{}\nstep-2 start\n'.format('='*60))
        self.findmd5 = FileMd5(self.result1_file,
                               msg_fun=self.disp_msg,
                               pbar=self.update_progress_bar)
        self.findmd5.run()
        self.result2_md5 = self.findmd5.find_file_md5_list
        self.write_result(task=self.task,
                          step=2,
                          result_seq=self.result2_md5)
        if self.bak_result:
            self.bak_result_fun(2)
        self.text_result.insert(tk.INSERT, 'step-2 end\n{}\n'.format('='*60))
        self.run_state = ''

    def run_prog3(self):
        self.disp_msg('{}\nstep-3 sart ...\n'.format('='*60))
        self.result2_file = self.read_result(task=self.task, step=2)
        if self.result2_file is None:
            self.run_state = ''
            return

        self.disp_msg('{}\nstep-3 group by md5 using iter-groupby...\n'.format('='*60))
        findgrp = FileGroupMd5(self.result2_file,
                               msg_fun=self.disp_msg,
                               pbar=self.update_progress_bar)
        findgrp.run_iter()
        self.result3_group = findgrp.find_group_list

        self.write_result(task=self.task,
                          step=3,
                          result_seq=self.result3_group)
        if self.bak_result:
            self.bak_result_fun(3)
        self.text_result.insert(tk.INSERT, 'step-3 end\n{}\n'.format('='*60))
        self.run_state = ''

    def run_prog4(self):
        # find result4(same files group) from file_group_md5(result3)
        self.result3_group = self.read_result(task=self.task, step=3)
        if self.result3_group is None:
            self.run_state = ''
            self.disp_msg('read result3 fail!')
            return
        self.disp_msg('{}\nstep-4: check same file group start'.format('='*60))
        findsame = FileGroupSame(self.result3_group,
                                 msg_fun=self.disp_msg,
                                 pbar=self.update_progress_bar)
        findsame.run()
        self.result4_same = findsame.same_group_list
        self.write_result(task=self.task,
                          step=4,
                          result_seq=self.result4_same)
        self.write_rsult4_report(task=self.task, group_list=self.result4_same)
        if self.bak_result:
            self.bak_result_fun(4)
        self.disp_msg('step-4 end\n{}\n'.format('='*60))
        self.run_state = ''

    def run_prog5(self):
        self.result4_same = self.read_result(task=self.task, step=4)
        if self.result4_same is None:
            self.run_state = ''
            return
        self.disp_msg('{}\nstep-5: move file start'.format('='*60))

        # process
        t = time.time()
        mover = FileMover(
            task=self.task,
            move_group_list=self.result4_same,
            move_option=self.move_option,
            move_path=self.move_path,
            move_suffix=self.move_suffix,
            move_num=self.move_num,
            disp_msg=self.disp_msg,
            pbar=self.update_progress_bar)
        mover.run()

        # result5
        result = mover.result_copy + mover.result_move + mover.result_fail
        self.write_result(self.task, step=5, result_seq=result)
        self.disp_msg('step-5 end\n{}\n'.format('='*60))
        self.run_state = ''

    def run_prog6(self):
        self.disp_msg('5-bakup start\n{}\n'.format('='*60))
        self.bak_result_fun(step=0)
        self.disp_msg('5-bakup end\n{}\n'.format('='*60))
        self.run_state = ''

    def update_progress_bar(self, percent=1, used_time=0, name='test'):
        hour = int(used_time / 3600)
        minute = int(used_time / 60) % 60
        second = used_time % 60
        self.canvas_progress_bar.itemconfig(self.canvas_text, text=name+' %02d:%02d:%02d' % (hour, minute, second))
        self.var_progress_bar_percent.set('%0.2f %%' % percent)
        green_length = int(self.bar_length * percent / 100)
        self.canvas_progress_bar.coords(self.canvas_shape, (0, 0, green_length, 25))

    def disp_msg(self, msg=''):
        if msg[-1] != '\n':
            msg += '\n'
        self.text_result.insert(tk.INSERT, msg)

    def write_result(self, task, step, result_seq):
        filename = task+'_'+'result_'+str(step)+'.csv'
        fp = open(filename, 'w', encoding='utf-8')
        fp.write(str(result_seq))
        fp.close()

    def bak_result_fun(self, step=1):
        step_num = 5
        step_all = 0
        task_path = self.task + '_bak/'
        if not os.path.isdir(task_path):
            os.mkdir(task_path)
        tm = time.localtime()
        frecord = '-'.join(map(str, [tm.tm_year, tm.tm_mon, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec]))
        _range = range(1, step_num+1) if step == step_all else [step]

        # bakup [task]_result_step.csv
        for _step in _range:
            step_str = str(_step)
            f_orig = self.task + '_result_' + step_str + '.csv'
            f_dest = task_path + self.task + '_result_' + step_str + '__' + frecord + '.csv'
            if os.path.isfile(f_orig):
                shutil.copyfile(f_orig, f_dest)
                self.disp_msg('done bakup result={}'.format(f_orig))
            else:
                self.disp_msg('fail bakup result={}'.format(f_orig))

        # bakup [task]_result_step_report
        for _step in _range:
            f_orig = self.task + '_result_' + str(_step) + '_report.csv'
            f_dest = task_path + self.task + '_result_' + str(_step) + '_report__' + frecord + '.csv'
            if os.path.isfile(f_orig):
                shutil.copyfile(f_orig, f_dest)
                self.disp_msg('bakup report={}'.format(f_orig))
            else:
                self.disp_msg('fail to bakup report={}'.format(f_orig))

    def write_rsult4_report(self, task, group_list: list):
        check_dir = self.check_path
        save_file = task + '_result_4_' + 'report.csv'
        same_file_group_list = sorted(group_list, key=lambda x: x[0][1])
        total_same_file_num = sum([len(g) for g in same_file_group_list])
        t = time.time()
        tl = time.localtime()
        self.disp_msg('write same files...')
        with open(save_file, 'w', encoding='utf-8') as fw:
            fw.write('check same files in [{}] at {}\n'.
                     format(check_dir, [tl.tm_year, tl.tm_mon, tl.tm_mday, tl.tm_hour, tl.tm_min, tl.tm_sec])
                     )
            fw.write('='*120 + '\n')
            fw.write('same files : {}\n'.format(total_same_file_num))
            fw.write('same groups: {}\n'.format(len(same_file_group_list)))
            fw.write('-'*120 + '\n')
            for fno, flist in enumerate(same_file_group_list):
                for i, ff in enumerate(flist):
                    if i == 0:
                        fw.write('[{:^5d}]  same number={}  filesize={}\n'.
                                 format(fno, len(flist), ff[1]))
                        fw.write('-' * 50 + '\n')
                    fw.write(' ' * 8 + ff[0] + '\n')
                if fno < len(same_file_group_list) - 1:
                    fw.write('-' * 50 + '\n')
            fw.write('-'*120 + '\n')
        self.disp_msg('write elapsed={:.3f}'.format(time.time()-t))

    def read_result(self, task, step):
        # t = time.time()
        filename = task+'_'+'result_'+str(step)+'.csv'
        if os.path.isfile(filename):
            self.disp_msg('reading {}_result_{}'.format(task, step))
            fp = open(filename, 'r', encoding='utf-8')
            result = fp.read()
            fp.close()
            # self.disp_msg('reading end {}'.format(time.time()-t))
            result = eval(result)
            # self.disp_msg('eval end {}'.format(time.time()-t))
            return result
        else:
            msgbox.showwarning('Step-' + str(step) + ' Warning',
                               'result file not found: {}'.format(filename))
            return None


class FileMover:
    def __init__(self, task, move_group_list, move_option, move_path, move_suffix, disp_msg, pbar, move_num=-1):
        self.task = task
        self.group_list = move_group_list
        self.option = move_option
        self.path = move_path
        self.suffix = move_suffix
        self.disp_msg = disp_msg
        self.pbar = pbar
        self.move_num = move_num
        # init path
        if not os.path.isdir(self.path):
            try:
                os.mkdir(self.path)
                disp_msg('create move path: {}'.format(self.path))
            except OSError:
                msgbox.showwarning('move files error', 'invalid path name: {}'.format(move_path))
                return
        # cut group_list
        if 0 < self.move_num < len(self.group_list):
            self.group_list = self.group_list[:self.move_num]
        disp_msg('files to move number={}'.format(len(self.group_list)))
        # set result
        self.result_fail = []
        self.result_copy = []
        self.result_move = []

    def run(self):
        # tlen = len(self.group_list)
        move_list = self.move_filter(group_list=self.group_list,
                                     suffix=self.suffix)
        # copy
        if self.option in ['1']:
            self.copy_delete(move_list, target_path=self.path, delete=False)
        # delete
        elif self.option in ['2']:
            self.copy_delete(move_list, target_path=self.path, delete=True)
        else:
            self.delete(move_list)
        # disp result
        self.disp_msg('copy files={}'.format(len(self.result_copy)))
        self.disp_msg('dele files={}'.format(len(self.result_move)))
        self.disp_msg('fail files={}'.format(len(self.result_fail)))
        # write result
        self.result_report()

    @staticmethod
    def move_filter(group_list, suffix):
        # suffix filter
        _suf = suffix.replace('.', '')
        if _suf == '*'*len(_suf):
            move_list = [[f[0] for f in fg] for fg in group_list]
        elif '*' in _suf:
            pt = _suf.replace('*', '.*')
            move_list = [[f[0] for f in fg if re.match(pt, f[0][f[0].rfind('.')+1:])]
                         for fg in group_list]
            return move_list
        else:
            move_list = [[f[0] for f in fg if f[0][f[0].rfind('.')+1:] == _suf]
                         for fg in group_list]
        move_list = [f for f in move_list if len(f) > 0]
        return move_list

    def copy_delete(self, move_list, target_path, delete=False):
        # copy proc
        move_fail = []
        t = time.time()
        tlen = len(move_list)
        for fi, fg in enumerate(move_list):
            self.pbar(percent=int(fi/tlen*100), used_time=time.time()-t, name='copy')
            # copy the longest name file in a group
            _fg = sorted(fg, key=lambda x: len(os.path.split(x)[1]))
            f_copy = _fg[-1]
            if os.path.isfile(f_copy):
                self.disp_msg('copy: ' + f_copy + ' ==> ' + target_path + '/')
                ff = os.path.split(f_copy)[1]
                try:
                    if delete:
                        shutil.move(f_copy, target_path + '/' + ff)
                        self.result_move.append('copy done:'+f_copy)
                        # delete else
                        for f in _fg[:-1]:
                            os.remove(f)
                            self.result_move.append('dele done:'+f)
                    else:
                        shutil.copyfile(f_copy, target_path + '/' + ff)
                        self.result_copy.append('copy done: '+f_copy)
                except IOError:
                    move_fail.append('move fail: ' + f_copy)
        self.disp_msg('copy end, elapsed={}'.format(time.time()-t))

    def delete(self, move_list):
        # longest name file reserved
        t = time.time()
        tlen = len(move_list)
        self.disp_msg('delete groups={} ...'.format(len(move_list)))
        for fi, fg in enumerate(move_list):
            self.pbar(percent=int(fi/tlen*100), used_time=time.time()-t, name='delete')
            # preserve long filename
            _fg = sorted(fg, key=lambda x: len(os.path.split(x)[1]))
            _fg = _fg[:-1]
            for _f in _fg:
                # delete
                try:
                    os.remove(_f)
                    self.result_move.append('move done: ' + _f)
                except OSError:
                    self.result_fail.append('move fail: ' + _f)
        self.disp_msg('deleted files={}, delete fail={}, elapsed={}'.
                      format(len(self.result_move),
                             len(self.result_fail),
                             time.time()-t))
        return

    def result_report(self):
        result_file = self.task+'_result_5_report.csv'
        with open(result_file, 'w', encoding='utf-8') as fp:
            for s in self.result_copy:
                fp.write(s+'\n')
            for s in self.result_move:
                fp.write(s+'\n')
            for s in self.result_fail:
                fp.write(s+'\n')


class FileFinder:
    def __init__(self, path='e:/test'):
        self.root_dir = path

        # result data
        self.find_dir_list = []
        self.find_file_list = []
        self.find_fail = []

        # result number
        self.total_size = 0
        self.total_file_num = 0
        self.find_dir_num = 0

    def run_subdir_files(self):
        self.total_size = 0
        self.total_file_num = 0
        self.find_dir_list = []
        self.find_file_list = []
        self.find_dir_num = 0
        print('get subdir files ...')
        # pbar = ProgressBar(total=10**6)
        t = time.time()
        self.get_subdir_files(self.root_dir)
        print('elapsed: {:3f}'.format(time.time()-t))
        return self.find_file_list

    def get_subdir_files(self, check_dir):
        self.find_dir_num += 1
        # pbar.log()
        this_size = 0
        try:
            this_check = os.listdir(check_dir)
        except IOError:
            self.find_fail.append('dir: '+check_dir)
            # this_check = []
            return

        for d in this_check:
            full_name = check_dir + '/' + d
            if os.path.isdir(full_name):
                self.get_subdir_files(full_name)
            else:
                this_size += os.path.getsize(full_name)
                self.find_file_list.append(full_name)
                self.total_file_num += 1
        self.find_dir_list.append([check_dir, this_size])
        self.total_size += this_size
        return


class FileMd5:
    def __init__(self, file_list=None, msg_fun=None, pbar=None):
        self.msg_fun = msg_fun
        self.pbar = pbar
        self.find_file_list = file_list

        # result data
        self.find_file_md5_list = []
        self.find_fail = []

        # result number
        self.total_size = 0
        self.total_file_num = 0
        self.total_group_num = 0

    def run(self):
        if len(self.find_file_list) == 0:
            self.msg_fun('no file in file_list!')
            return
        else:
            self.msg_fun('-'*60+'\n')
        self.find_file_md5_list = self.run_md5_thread(self.find_file_list)
        self.find_file_md5_list = sorted(self.find_file_md5_list, key=lambda x: x[2])

    def run_md5_thread(self, find_file_list):
        t = time.time()
        seg_num = os.cpu_count()
        self.total_file_num = len(find_file_list)
        seg_len = int(self.total_file_num/seg_num)
        file_seg_list = [find_file_list[i*seg_len:(i+1)*seg_len] if i < seg_num-1 else
                         find_file_list[i*seg_len:]
                         for i in range(seg_num)]
        with futures.ThreadPoolExecutor(max_workers=seg_num) as executor:
            to_do = []
            future_dict = dict()
            for fi, file_list in enumerate(file_seg_list):
                future = executor.submit(self.run_md5, file_list, 'md5 thread-'+str(fi))
                to_do.append(future)
                future_dict.update({future: fi})
            self.msg_fun('-'*60+'\n')
            result = []
            for future in futures.as_completed(to_do):
                res = future.result()
                result.extend(res)
                # self.msg_fun('md5 thread-{} end ... \n'.format(fi))
        self.msg_fun('{1}\ncalc md5 end  elapsed={0:.2f} \n'.format(time.time()-t, '-'*60))
        return result

    def run_md5(self, file_list=None, task='run md5'):
        t = time.time()
        self.msg_fun('{}\n'.format(task))
        if len(file_list) == 0:
            self.msg_fun('no file found!\n')
            return

        find_file_md5_list = []
        total = len(file_list)
        for i, f in enumerate(file_list):
            percent = int(i/total*100)
            self.pbar(percent, time.time()-t, task)
            try:
                f_m5 = self.make_md5(f)
            except IOError:
                self.find_fail.append('file: ' + f)
                continue
            find_file_md5_list.append([f, os.path.getsize(f), f_m5])
        self.msg_fun(task+'end  elapsed: {:3f}\n'.format(time.time()-t))
        return find_file_md5_list

    @staticmethod
    def make_md5(filename):
        # block_size = 10*1024 * 1024
        m5 = hashlib.md5()
        with open(filename, 'rb') as fp:
            m5.update(fp.read())
            # while True:
            #     data = fp.read(block_size)
            #     if not data:
            #         break
            #     m5.update(data)
        return m5.digest()


class FileGroupMd5:
    def __init__(self, md5_list=None, msg_fun=None, pbar=None):
        self.msg_fun = msg_fun
        self.pbar = pbar
        self.md5_list = md5_list

        # result data
        self.find_group_list = []
        self.find_fail = []

    def run(self):
        tstart = time.time()
        self.msg_fun('get md5 group ...')

        # create groups with same files
        file_size_md5_list = self.md5_list
        file_group_same_md5 = []
        current_group = []
        last_md5 = b''
        total_len = len(file_size_md5_list)
        for fi, f_size_md5 in enumerate(file_size_md5_list):
            self.pbar(int(fi/total_len*100), time.time()-tstart, 'group by md5 ')
            if f_size_md5[2] != last_md5:
                # add group to file_group if 2 more files in
                if len(current_group) > 1:
                    file_group_same_md5.append(current_group)
                # new group
                current_group = [f_size_md5]  # [ftuple(f_name_md5[0], os.path.getsize(f_name_md5[0]), f_name_md5[2])]
                last_md5 = f_size_md5[2]
            else:
                # current_group.append(ftuple(f_name_md5[0], os.path.getsize(f_name_md5[0]), f_name_md5[2]))
                current_group.append(f_size_md5)
        # group with 2 more files
        if len(current_group) > 1:
            file_group_same_md5.append(current_group)

        self.msg_fun('md5 group end,  elapsed={:.3f}\n'.format(time.time()-tstart))
        self.find_group_list = file_group_same_md5

    def run_iter(self):
        import itertools as itl
        tstart = time.time()
        self.msg_fun('get md5 group ...')
        # create groups with same files
        # file_size_md5_list = sorted(self.md5_list, key=lambda x: x[2])
        file_size_md5_list = self.md5_list  # sorting finished in FileMd5.run
        group = itl.groupby(file_size_md5_list, key=lambda x: x[2])
        result_group = []
        for k, fg in group:
            file_list = list(fg)
            if len(file_list) > 1:
                result_group.append(file_list)
        self.msg_fun('get md5 group end, elapsed={}'.format(time.time()-tstart))
        self.find_group_list = result_group


class FileGroupSame:
    def __init__(self, md5_group_list=None, msg_fun=None, pbar=None):
        self.msg_fun = msg_fun
        self.pbar = pbar
        self.md5_group_list = md5_group_list

        # result data
        self.same_group_list = []
        self.find_fail = []

    def run(self):
        tstart = time.time()
        self.msg_fun('get same content group start ...')
        workers = os.cpu_count()
        seg_len = int(len(self.md5_group_list)/workers)
        file_seg_list = [self.md5_group_list[i*seg_len:(i+1)*seg_len]
                         if i < workers-1 else
                         self.md5_group_list[i*seg_len:]
                         for i in range(workers)]
        with futures.ThreadPoolExecutor(max_workers=workers) as executor:
            to_do = []
            future_dict = dict()
            for fi, fglist in enumerate(file_seg_list):
                future = executor.submit(self.run_get_groups_same_content, fglist)
                to_do.append(future)
                future_dict.update({future: fi})
                self.msg_fun('check same group thread-{} start ...'.format(fi))
            result = []
            for future in futures.as_completed(to_do):
                res = future.result()
                result.extend(res)
                self.msg_fun('thread-{} end'.format(future_dict[future]))
        self.msg_fun('check same content total elapsed: {:.2f}'.format(time.time()-tstart))
        self.same_group_list = result
        # return result

    def run_get_groups_same_content(self, file_group_same_md5):
        t = time.time()
        file_group_same_content = []
        tlen = len(file_group_same_md5)
        for i, fg in enumerate(file_group_same_md5):
            self.pbar(int(i/tlen*100), time.time()-t, '')
            if len(fg) <= 1:
                continue
            elif len(fg) == 2:
                if fg[0][2] == fg[1][2]:
                    # size == 0
                    if fg[0][1] == 0:
                        file_group_same_content.append(fg)
                        continue
                    with open(fg[0][0], 'rb') as f1, open(fg[1][0], 'rb') as f2:
                        f1r = f1.read()
                        f2r = f2.read()
                        if f1r == f2r:
                            file_group_same_content.append(fg)
                continue
            # len(fg) > 2
            else:
                same_group = []
                same_to_last = False
                for fi, f in enumerate(fg[:-1]):
                    # compare by md5
                    if f[2] == fg[fi+1][2]:
                        if same_to_last:
                            same_group.append(f)
                        else:
                            if len(same_group) > 0:
                                file_group_same_content.append(same_group)
                            same_group = [f]
                        same_to_last = True
                        # last item
                        if fi == len(fg) - 2:
                            same_group.append(fg[fi+1])
                            file_group_same_content.append(same_group)
                    else:
                        if same_to_last:
                            same_group.append(f)
                            file_group_same_content.append(same_group)
                            same_group = []
                        else:
                            # single file, different from last and next
                            # save and clear same_group
                            if len(same_group) > 1:
                                file_group_same_content.append(same_group)
                                same_group = []
                        same_to_last = False
        return file_group_same_content


class Document:
    help_doc = \
        """
        【基本说明】

        本软件用于检测一个目录中的重复文件, 分为1,2,3,4,5个功能:
        
        1.find   [step-1]: 获取路径文件名

        2.md5    [step-2]: 计算每个文件内容md5值

        3.group  [step-3]: 按文件的md5值进行分组

        4.result [step-4]: 对分组内文件进行内容比较，产生相同内容的文件的分组

        5.move   [step-5]: 移除重复文件（1-只拷贝重复文件到目标目录，使用最长文件名作为目标文件名
                                      
                                         2-拷贝重复文件到目标目录，并删除原目录中文件，保留原目录中最长文件名文件
                                      
                                         3-只删除原目录中重复文件，保留原目录中最长文件名文件）

        6.bakup  [step-6]: 备份前面5个步骤的运行结果
        
        在1-5步骤中，每一步骤读取上一步的结果，并单独保存本步骤的运行结果[task]_result_[step].csv.
        
        如果选择了bak_result/2-bakup result, 则每个步骤都在运行后备份结果及4-5步骤的报告report。
        
        【输入参数】
        
        task: 检测任务名称，根据情况设定用以表示各类输出结果，不能包含非文件名、路径名合法字符
        
        path: 检查相同内容文件的目录名，需要带有完整路径名称
        
        bakresult: 是否同时备份当前运行步骤的运行结果（[task]_result_[step]...)
        
        move_option: 移除相同文件的方式
        
        move_path: 将相同内容的文件移除到的目标目录
        
        move_suffix: 移除文件后缀（只处理该后缀名的文件），支持使用‘*’匹配，如，‘*ab'指末尾为ab的后缀，’*'为所有名称

        【输出结果】
        
        步骤结果： [task]_result_[step].csv，step = 1,2,3,4,5
        
        步骤报告： task]_result_[[step]_report.csv, step = 4,5

        备份数据： copy [task]_result_[[step]_report.csv
                   to   [task]_bak/[task]_result_..._[year-mon-mday-hour-min-sec].csv
        
        """


if __name__ == '__main__':
    root = Tk()
    w = MainWin(root)
    root.mainloop()
